# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from Spider80Text.items import *


class SpiderallSpider(CrawlSpider):
    name = 'SpiderAll'
    allowed_domains = ['80txt.com']
    start_urls = ['https://www.80txt.com/txtxz/77050.html']

    rules = (
        #Rule(LinkExtractor(allow=r'https://www.80txt.com/sort'), callback='parse_class', follow=True),
        #Rule(LinkExtractor(allow=r'https://www.80txt.com/txtxz'), callback='parse_book', follow=True),
        Rule(LinkExtractor(allow=r'https://www.80txt.com/txtml'), callback='parse_bookCatalog', follow=True),
        #Rule(LinkExtractor(allow=r'https://www.80txt.com/txtml'), callback='章节', follow=True),
    )

    def parse_class(self, response):
        divlist = response.xpath('//*[@id="list_art_2013"]')
        items = []
        for div in divlist:
            item = Spider80TextItem()
            item['bookName'] = div.xpath('div[2]/a/img/@title').extract()[0]
            item['bookUrl'] = div.xpath('div[1]/div[1]/div[1]/a/@href').extract()[0]
            item['bookAuthor'] = div.xpath('div[1]/div[3]/span/a/text()').extract()[0]
            items.append(item)
        return items

    def parse_book(self, response):
        item = Spider80TextbookItem()
        item['bookUrl'] = response.xpath('//*[@id="read_book"]/@href').extract()[1]
        item['bookName'] = response.xpath('//*[@id="soft_info_para"]/h1/text()').extract()[0]
        return item

    def parse_bookCatalog(self, response):
        divlist = response.xpath('//*[@id="yulan"]/li')
        items = []
        for div in divlist:
            item = BookCatalog()
            url = str(response.url);
            searchObj = re.search( r'https://www.80txt.com/txtml_(.*?).html', url, re.M|re.I)
            item['Id'] = searchObj.group(1)
            item['URL'] = div.xpath('a/@href').extract()[0]
            item['Chapter'] = div.xpath('a/text()').extract()[0]
            items.append(item)
        return items



